import urllib.request as ur
import lxml.etree as le
import user_agent


# 加入用户代理客户端
def get_url_client(url):
    url_client = ur.Request(
        url=url,
        headers={
            'User-Agent': user_agent.get_user_agent_pc(),
            'Cookie': 'acw_tc=2760822d15745757923936720e7f1eb906432689038f43999eee9d6d8e5ca4; acw_sc__v2=5dda1eb00897c50f50abf21415c0693a04bbfdd1; uuid_tt_dd=10_2430070490-1574575792566-423120; dc_session_id=10_1574575792566.141905; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1574575792; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1574575792; Hm_ct_6bcd52f51e9b3dce32bec4a3997715ac=6525*1*10_2430070490-1574575792566-423120; acw_sc__v3=5dda1eb15c60145d5f920462fb8ebacd0ca247c8; c-login-auto=1; announcement=%257B%2522isLogin%2522%253Afalse%252C%2522announcementUrl%2522%253A%2522https%253A%252F%252Fblogdev.blog.csdn.net%252Farticle%252Fdetails%252F103053996%2522%252C%2522announcementCount%2522%253A0%252C%2522announcementExpire%2522%253A3600000%257D; dc_tos=q1gmgh'
        }
    )
    return url_client


# 加入用户代理IP
def get_user_ip():
    # 购买代理
    ip_address = ur.urlopen(
        'http://api.ip.data5u.com/dynamic/get.html?order=e63963b7c5ce86c0ae4575e6cad0390b&sep=4')
    ip_handler = ur.ProxyHandler(
        {
            'http': ip_address
        }
    )
    return ur.build_opener(ip_handler)


# 创建主函数
def pc_main(kw, pg_start, pg_end):
    for pg in range(pg_start, pg_end + 1):
        pc_request = get_url_client(
            'https://so.csdn.net/so/search/s.do?p=%s&q=%s&t=blog&viparticle=&domain=&o=&s=&u=&l=&f=&rbg=0' % (pg, kw)
        )
        try:
            pc_response = get_user_ip().open(pc_request).read()
            href_s = le.HTML(pc_response).xpath('//span[@class="down fr"]/../span[@class="link"]/a/@href')
            for href in href_s:
                try:
                    blog_request = get_url_client(href)
                    blog_response = get_user_ip().open(blog_request).read()
                    title = le.HTML(blog_response).xpath('//h1[@class="title-article"]/text()')[0]
                    with open('blog/%s.html' % title, 'wb') as f:
                        f.write(blog_response)
                except Exception as e:
                    with open('blog/log.txt', 'w') as f:
                        f.write(e)

        except Exception as e:
            print(e)


if __name__ == '__main__':
    kw = input('请输入关键字：')
    pg_start = int(input('抓取的起始页：'))
    pg_end = int(input('抓取的结束页：'))
    pc_main(kw, pg_start, pg_end)
